Churn prevention allows companies to develop loyalty programs and retention campaigns to keep as many customers as possible.

In this example, we use customer data from a bank to construct a predictive model for the likely churn clients.

The variable to be predicted is binary (churn or loyal). Therefore this is a classification project.

The goal here is to model churn probability, conditioned on the customer features.

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import style
import seaborn as sns
sns.set_style('whitegrid')
from sklearn.model_selection import train_test_split, cross_val_predict
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.decomposition import PCA
from xgboost import XGBClassifier
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score
import time
import warnings
warnings.filterwarnings("ignore")

Import data

data is from a modelling competition and it contain sensitive data

df_train.head()

	x0	x1	x2	x3	x4	x5	x6	x7	x8	x9	...	x116	x117	x118	x119	x120	x121	x122	x123	x124	y
0	0.067563	-2.172492	-1.713774	-0.146854	-0.685219	-0.933838	4.0	2.0	-4.476051	-1.472306	...	4.607926	52.0	1.0	0.773809	0.766531	6.307142	9.074515	-2.694049	0.330964	0
1	0.099622	0.584803	-6.089758	-0.252186	-0.334912	-4.671281	4.0	2.0	-0.387076	-0.664552	...	15.889763	53.0	1.0	7.065852	0.994304	-0.028411	-21.472850	5.626095	1.293455	0
2	0.648421	3.881137	0.961359	0.217361	-1.471850	-0.975352	5.0	1.0	-0.850146	-4.011487	...	13.770410	53.0	1.0	-8.598553	-2.958302	-12.784030	-36.855873	0.356093	1.499790	0
3	0.005775	-2.726153	2.891378	-0.038325	2.280847	1.761478	3.0	3.0	9.116188	-4.610151	...	5.904901	50.0	1.0	8.194975	-7.798308	10.910515	15.226094	-10.440813	-0.735116	0
4	0.044396	2.803576	-1.923381	-0.116657	-1.574199	6.098627	3.0	1.0	-13.956141	13.785099	...	-14.603703	47.0	0.0	2.330789	-8.249825	-12.680490	40.192302	-5.037065	1.679262	1

5 rows × 126 columns

df_test.head()

	x0	x1	x2	x3	x4	x5	x6	x7	x8	x9	...	x116	x117	x118	x119	x120	x121	x122	x123	x124	y
0	0.403735	-1.966104	-1.322339	0.084642	2.481997	-1.719155	5.0	2.0	-11.546670	-7.073770	...	-7.765934	59.0	1.0	-9.897690	5.694125	-6.734618	17.052251	3.601040	2.873498	0
1	0.032128	6.516294	-0.939752	0.026937	-0.992999	5.995462	4.0	3.0	-5.634876	1.937998	...	15.667015	47.0	1.0	7.781887	9.752716	5.918369	15.922908	5.460557	10.475816	0
2	-0.280345	0.930291	-1.028828	-0.050924	-3.164733	6.885108	3.0	2.0	1.609005	-1.812488	...	8.523648	55.0	0.0	6.815214	-3.325532	4.123546	-19.656038	8.380318	-1.203315	0
3	-0.313420	-0.568857	-1.576051	-0.003607	5.539969	-1.594770	4.0	3.0	-11.517734	-5.364282	...	-13.912115	41.0	0.0	3.158137	-7.088875	-3.035983	34.277048	-3.169203	1.417681	0
4	0.108825	-4.815631	5.491084	-0.123928	-1.932024	4.014821	5.0	3.0	-3.117576	2.004889	...	-14.432712	52.0	0.0	-3.867657	-13.308519	1.008302	41.449381	-0.804560	-0.649453	0

5 rows × 126 columns

EDA

df_train.groupby('x21').agg({"x0":(np.nanmean,np.nanstd),"x1":np.nanmean})

	x0		x1
	nanmean	nanstd	nanmean
x21
Apr	-0.004327	0.421969	-0.000844
Aug	0.003092	0.424122	0.045679
Dec	-0.163770	0.390685	0.907038
Feb	-0.040469	0.412641	-0.053096
Jan	0.036535	0.319520	0.751807
Jul	0.000136	0.423281	-0.007223
Jun	0.001005	0.425301	0.003522
Mar	0.004019	0.428786	-0.218786
May	-0.002314	0.423925	0.014539
Nov	0.025535	0.392620	0.158384
Oct	0.011870	0.432944	-0.049502
Sep	-0.003915	0.428036	-0.076729

df_train.dtypes

x0      float64
x1      float64
x2      float64
x3      float64
x4      float64
         ...   
x121    float64
x122    float64
x123    float64
x124    float64
y         int64
Length: 126, dtype: object

null_val_sums = df_train.isnull().sum()
pd.set_option("display.max_rows", None)
pd.DataFrame({"Column": null_val_sums.index, "Number of Null Values": null_val_sums.values,
             "Proportion": null_val_sums.values / len(df_train) })

	Column	Number of Null Values	Proportion
0	x0	123	0.00123
1	x1	131	0.00131
2	x2	124	0.00124
3	x3	115	0.00115
4	x4	137	0.00137
5	x5	116	0.00116
6	x6	122	0.00122
7	x7	133	0.00133
8	x8	134	0.00134
9	x9	130	0.00130
10	x10	142	0.00142
11	x11	128	0.00128
12	x12	135	0.00135
13	x13	145	0.00145
14	x14	156	0.00156
15	x15	123	0.00123
16	x16	138	0.00138
17	x17	129	0.00129
18	x18	136	0.00136
19	x19	123	0.00123
20	x20	131	0.00131
21	x21	126	0.00126
22	x22	136	0.00136
23	x23	127	0.00127
24	x24	132	0.00132
25	x25	116	0.00116
26	x26	141	0.00141
27	x27	134	0.00134
28	x28	141	0.00141
29	x29	114	0.00114
30	x30	119	0.00119
31	x31	130	0.00130
32	x32	123	0.00123
33	x33	135	0.00135
34	x34	138	0.00138
35	x35	115	0.00115
36	x36	138	0.00138
37	x37	139	0.00139
38	x38	124	0.00124
39	x39	132	0.00132
40	x40	127	0.00127
41	x41	114	0.00114
42	x42	137	0.00137
43	x43	133	0.00133
44	x44	131	0.00131
45	x45	159	0.00159
46	x46	126	0.00126
47	x47	121	0.00121
48	x48	134	0.00134
49	x49	117	0.00117
50	x50	141	0.00141
51	x51	111	0.00111
52	x52	122	0.00122
53	x53	137	0.00137
54	x54	109	0.00109
55	x55	125	0.00125
56	x56	118	0.00118
57	x57	112	0.00112
58	x58	123	0.00123
59	x59	130	0.00130
60	x60	127	0.00127
61	x61	139	0.00139
62	x62	108	0.00108
63	x63	124	0.00124
64	x64	130	0.00130
65	x65	137	0.00137
66	x66	127	0.00127
67	x67	121	0.00121
68	x68	126	0.00126
69	x69	141	0.00141
70	x70	159	0.00159
71	x71	127	0.00127
72	x72	149	0.00149
73	x73	126	0.00126
74	x74	138	0.00138
75	x75	146	0.00146
76	x76	119	0.00119
77	x77	124	0.00124
78	x78	139	0.00139
79	x79	118	0.00118
80	x80	137	0.00137
81	x81	101	0.00101
82	x82	117	0.00117
83	x83	120	0.00120
84	x84	139	0.00139
85	x85	139	0.00139
86	x86	141	0.00141
87	x87	131	0.00131
88	x88	124	0.00124
89	x89	125	0.00125
90	x90	120	0.00120
91	x91	138	0.00138
92	x92	137	0.00137
93	x93	125	0.00125
94	x94	109	0.00109
95	x95	132	0.00132
96	x96	130	0.00130
97	x97	115	0.00115
98	x98	132	0.00132
99	x99	122	0.00122
100	x100	126	0.00126
101	x101	114	0.00114
102	x102	124	0.00124
103	x103	141	0.00141
104	x104	114	0.00114
105	x105	124	0.00124
106	x106	108	0.00108
107	x107	130	0.00130
108	x108	138	0.00138
109	x109	130	0.00130
110	x110	148	0.00148
111	x111	127	0.00127
112	x112	142	0.00142
113	x113	104	0.00104
114	x114	100	0.00100
115	x115	120	0.00120
116	x116	123	0.00123
117	x117	136	0.00136
118	x118	127	0.00127
119	x119	122	0.00122
120	x120	120	0.00120
121	x121	129	0.00129
122	x122	127	0.00127
123	x123	130	0.00130
124	x124	123	0.00123
125	y	0	0.00000

null_val_sums = df_test.isnull().sum()
pd.set_option("display.max_rows", None)
pd.DataFrame({"Column": null_val_sums.index, "Number of Null Values": null_val_sums.values,
             "Proportion": null_val_sums.values / len(df_test) })

	Column	Number of Null Values	Proportion
0	x0	40	0.00160
1	x1	30	0.00120
2	x2	26	0.00104
3	x3	27	0.00108
4	x4	39	0.00156
5	x5	36	0.00144
6	x6	36	0.00144
7	x7	31	0.00124
8	x8	32	0.00128
9	x9	38	0.00152
10	x10	24	0.00096
11	x11	35	0.00140
12	x12	27	0.00108
13	x13	46	0.00184
14	x14	26	0.00104
15	x15	30	0.00120
16	x16	39	0.00156
17	x17	32	0.00128
18	x18	27	0.00108
19	x19	40	0.00160
20	x20	39	0.00156
21	x21	34	0.00136
22	x22	31	0.00124
23	x23	31	0.00124
24	x24	39	0.00156
25	x25	39	0.00156
26	x26	27	0.00108
27	x27	39	0.00156
28	x28	26	0.00104
29	x29	21	0.00084
30	x30	37	0.00148
31	x31	24	0.00096
32	x32	19	0.00076
33	x33	45	0.00180
34	x34	27	0.00108
35	x35	30	0.00120
36	x36	40	0.00160
37	x37	32	0.00128
38	x38	26	0.00104
39	x39	35	0.00140
40	x40	42	0.00168
41	x41	34	0.00136
42	x42	44	0.00176
43	x43	30	0.00120
44	x44	38	0.00152
45	x45	34	0.00136
46	x46	35	0.00140
47	x47	31	0.00124
48	x48	39	0.00156
49	x49	33	0.00132
50	x50	21	0.00084
51	x51	33	0.00132
52	x52	30	0.00120
53	x53	20	0.00080
54	x54	33	0.00132
55	x55	34	0.00136
56	x56	29	0.00116
57	x57	27	0.00108
58	x58	27	0.00108
59	x59	35	0.00140
60	x60	21	0.00084
61	x61	40	0.00160
62	x62	41	0.00164
63	x63	31	0.00124
64	x64	33	0.00132
65	x65	36	0.00144
66	x66	38	0.00152
67	x67	37	0.00148
68	x68	32	0.00128
69	x69	39	0.00156
70	x70	23	0.00092
71	x71	32	0.00128
72	x72	38	0.00152
73	x73	26	0.00104
74	x74	38	0.00152
75	x75	27	0.00108
76	x76	22	0.00088
77	x77	41	0.00164
78	x78	46	0.00184
79	x79	31	0.00124
80	x80	43	0.00172
81	x81	28	0.00112
82	x82	26	0.00104
83	x83	31	0.00124
84	x84	21	0.00084
85	x85	45	0.00180
86	x86	38	0.00152
87	x87	35	0.00140
88	x88	19	0.00076
89	x89	39	0.00156
90	x90	31	0.00124
91	x91	33	0.00132
92	x92	24	0.00096
93	x93	31	0.00124
94	x94	32	0.00128
95	x95	32	0.00128
96	x96	33	0.00132
97	x97	43	0.00172
98	x98	43	0.00172
99	x99	34	0.00136
100	x100	35	0.00140
101	x101	32	0.00128
102	x102	24	0.00096
103	x103	35	0.00140
104	x104	32	0.00128
105	x105	22	0.00088
106	x106	31	0.00124
107	x107	25	0.00100
108	x108	29	0.00116
109	x109	35	0.00140
110	x110	37	0.00148
111	x111	34	0.00136
112	x112	28	0.00112
113	x113	31	0.00124
114	x114	33	0.00132
115	x115	32	0.00128
116	x116	29	0.00116
117	x117	33	0.00132
118	x118	35	0.00140
119	x119	26	0.00104
120	x120	38	0.00152
121	x121	39	0.00156
122	x122	34	0.00136
123	x123	46	0.00184
124	x124	42	0.00168
125	y	0	0.00000

print("Proportion : {}".format(df_train["y"].sum() / len(df_train)))
sns.countplot(x='y', data=df_train)

Proportion : 0.16992

<AxesSubplot:xlabel='y', ylabel='count'>

categorical = df_train.columns[(df_train.dtypes.values != np.dtype('float64'))]

categorical

Index(['x21', 'x79', 'x89', 'x108', 'x112', 'y'], dtype='object')

numerical = df_train.columns[(df_train.dtypes.values == np.dtype('float64'))]

numerical

Index(['x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9',
       ...
       'x115', 'x116', 'x117', 'x118', 'x119', 'x120', 'x121', 'x122', 'x123',
       'x124'],
      dtype='object', length=120)

target_0 = df_train[df_train['y'] == 0]
target_1 = df_train[df_train['y'] == 1]

num_nonfloat = np.array(['x6','x7','x22','x51','x63','x80','x93','x96','x104','x117','x118']).astype('object')

num_nonfloat

array(['x6', 'x7', 'x22', 'x51', 'x63', 'x80', 'x93', 'x96', 'x104',
       'x117', 'x118'], dtype=object)

indx = np.ravel([np.where(numerical == i) for i in num_nonfloat])
numerical=np.delete(numerical,indx)

numerical

Index(['x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x8', 'x9', 'x10', 'x11',
       ...
       'x113', 'x114', 'x115', 'x116', 'x119', 'x120', 'x121', 'x122', 'x123',
       'x124'],
      dtype='object', length=109)

Categorical Data

for i in categorical:
    pd.crosstab(df_train[i],df_train['y']).plot(kind="bar")

for x in categorical:
    print(df_train[x].value_counts(),'\n')

Jun    28155
Jul    25018
May    18494
Aug    13334
Apr     7046
Sep     4738
Mar     1495
Oct     1160
Nov      197
Feb      183
Jan       27
Dec       27
Name: x21, dtype: int64 

Perempuan    50046
Laki-laki    49836
Name: x79, dtype: int64 

Kalimantan Tengah     12449
Kalimantan Barat      12343
Kalimantan Timur      11678
Sumatera Utara        10943
Kalimantan Selatan     9921
Sumatera Selatan       8715
Kalimantan Utara       7519
Sumatera Barat         6086
Gorontalo              4991
Jambi                  3926
Sulawesi Barat         3099
Bengkulu               2465
Sulawesi Tengah        1757
Bangka Belitung        1313
Sulawesi Tenggara       929
Jawa Timur              592
Sulawesi Utara          404
Jawa Tengah             284
Bali                    183
Jawa Barat              121
NTB                      61
DI Yogyakarta            41
NTT                      26
Banten                   11
Maluku                    9
Maluku Utara              6
DKI Jakarta               3
Name: x89, dtype: int64 

SMP              60285
SMA/SMK          23334
SD               15166
S1/S2/S3           773
Tidak sekolah      304
Name: x108, dtype: int64 

Jul    27476
Jun    25591
Aug    17806
May    14682
Sep     6668
Apr     4747
Oct     1550
Mar      991
Nov      218
Feb      111
Dec       16
Jan        2
Name: x112, dtype: int64 

0    83008
1    16992
Name: y, dtype: int64 

# for x in categorical:
#     df_train[x]=df_train[x].fillna(df_train[x].mode()[0],inplace=True)

temp = ['x22','x117']
for x in temp:
    sns.distplot(target_0[x],color='red',label='y = 0')
    sns.distplot(target_1[x],color='blue',label = 'y = 1')
    plt.legend(loc="upper right")
    min_ylim, max_ylim = plt.ylim()
    plt.axvline(target_0[x].mean(), color='k', linestyle='dashed', linewidth=1)
    plt.text(target_0[x].mean()*1.1, max_ylim*0.9, 'Mean: {:.2f}'.format(target_0[x].mean()),color='red')
    plt.axvline(target_1[x].mean(), color='k', linestyle='dashed', linewidth=1)
    plt.text(target_1[x].mean()*1.1, max_ylim*0.5, 'Mean: {:.2f}'.format(target_1[x].mean()),color= 'blue')
    plt.legend(loc="upper right")
    plt.show()

Non-Float Numerical Data

for i in num_nonfloat:
    pd.crosstab(df_train[i],df_train['y']).plot(kind="bar")

# for x in num_nonfloat:
#     print(x,(df_train[x].median()))
#     df_train[x].fillna(df_train[x].mode()[0],inplace=True)

Float Numerical Data

for x in numerical:
    sns.distplot(target_0[x],color='red',label='y = 0')
    sns.distplot(target_1[x],color='blue',label = 'y = 1')
    plt.legend(loc="upper right")
    min_ylim, max_ylim = plt.ylim()
    plt.axvline(target_0[x].median(), color='k', linestyle='dashed', linewidth=1)
    plt.text(target_0[x].mean()*1.1, max_ylim*0.9, 'Mean: {:.2f}'.format(target_0[x].mean()),color='red')
    plt.axvline(target_1[x].median(), color='k', linestyle='dashed', linewidth=1)
    plt.text(target_1[x].mean()*1.1, max_ylim*0.5, 'Mean: {:.2f}'.format(target_1[x].mean()),color= 'blue')
    plt.legend(loc="upper right")
    plt.show()

Feature Engineering

Impute Missing Value

Fill Missing Value Numerical Float with median

for x in numerical:
    df_train[x]=df_train[x].fillna(df_train[x].median())

null_val_sums = df_train[numerical].isnull().sum()
pd.set_option("display.max_rows", None)
pd.DataFrame({"Column": null_val_sums.index, "Number of Null Values": null_val_sums.values,
             "Proportion": null_val_sums.values / len(df_train[numerical]) })

Fill Missing value of Categorical and Numerical Non-float with Mode

for x in num_nonfloat:
    df_train[x] = df_train[x].fillna(df_train[x].mode()[0])

for x in categorical:
    df_train[x] = df_train[x].fillna(df_train[x].mode()[0])

null_val_sums = df_train[num_nonfloat].isnull().sum()
pd.set_option("display.max_rows", None)
pd.DataFrame({"Column": null_val_sums.index, "Number of Null Values": null_val_sums.values,
             "Proportion": null_val_sums.values / len(df_train[num_nonfloat]) })

Train-Test Split

X = df_train.drop(labels=['y'],axis=1)
y = df_train['y']

# from sklearn.model_selection import train_test_split
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

PCA

from sklearn.decomposition import PCA

# from sklearn.preprocessing import MinMaxScaler
# scaler = MinMaxScaler()
# X_rescaled = scaler.fit_transform(X[numerical])

pca = PCA().fit(X[numerical])
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')

Text(0, 0.5, 'cumulative explained variance')

pca.components_.shape

(109, 109)

cumvar = pd.DataFrame({'number of components': np.arange(1,109), 'explained variance ratio': np.cumsum(pca.explained_variance_ratio_)[1:]})
cumvar

	number of components	explained variance ratio
0	1	0.364404
1	2	0.488584
2	3	0.592564
3	4	0.669153
4	5	0.728714
5	6	0.784688
6	7	0.823808
7	8	0.860424
8	9	0.884802
9	10	0.903824
10	11	0.918579
11	12	0.927970
12	13	0.932245
13	14	0.935390
14	15	0.938453
15	16	0.941456
16	17	0.944326
17	18	0.947034
18	19	0.949636
19	20	0.952211
20	21	0.954771
21	22	0.957191
22	23	0.959443
23	24	0.961603
24	25	0.963747
25	26	0.965852
26	27	0.967853
27	28	0.969824
28	29	0.971728
29	30	0.973404
30	31	0.975042
31	32	0.976546
32	33	0.978041
33	34	0.979517
34	35	0.980892
35	36	0.982180
36	37	0.983402
37	38	0.984442
38	39	0.985423
39	40	0.986395
40	41	0.987313
41	42	0.988137
42	43	0.988917
43	44	0.989635
44	45	0.990341
45	46	0.991022
46	47	0.991666
47	48	0.992267
48	49	0.992849
49	50	0.993426
50	51	0.993972
51	52	0.994509
52	53	0.995027
53	54	0.995528
54	55	0.996020
55	56	0.996507
56	57	0.996937
57	58	0.997312
58	59	0.997673
59	60	0.998032
60	61	0.998334
61	62	0.998629
62	63	0.998770
63	64	0.998892
64	65	0.999003
65	66	0.999111
66	67	0.999197
67	68	0.999275
68	69	0.999348
69	70	0.999417
70	71	0.999474
71	72	0.999525
72	73	0.999574
73	74	0.999619
74	75	0.999659
75	76	0.999697
76	77	0.999730
77	78	0.999759
78	79	0.999786
79	80	0.999810
80	81	0.999831
81	82	0.999851
82	83	0.999869
83	84	0.999885
84	85	0.999898
85	86	0.999911
86	87	0.999923
87	88	0.999933
88	89	0.999941
89	90	0.999948
90	91	0.999954
91	92	0.999960
92	93	0.999966
93	94	0.999971
94	95	0.999975
95	96	0.999979
96	97	0.999983
97	98	0.999986
98	99	0.999989
99	100	0.999992
100	101	0.999994
101	102	0.999995
102	103	0.999997
103	104	0.999998
104	105	0.999998
105	106	0.999999
106	107	1.000000
107	108	1.000000

pca = PCA(n_components=35)
X_pca = pca.fit_transform(X[numerical])
# X_pca = pca.fit_transform(X_rescaled)
X_pca

array([[-1.64268682e+02, -1.04438239e+02,  4.59682238e+01, ...,
         1.09938465e+01,  1.25721029e+00,  4.58807968e+00],
       [ 7.27039442e+00,  1.08822932e+02, -2.30076889e+01, ...,
        -1.26302879e+01,  5.77186940e+00,  1.48875616e-01],
       [ 4.55537436e+01,  5.65109359e+01,  9.44267842e+00, ...,
        -2.42999265e+00, -7.03701477e+00,  1.04431395e+01],
       ...,
       [ 1.93014348e+01,  9.30277271e+01, -1.19168253e+01, ...,
        -7.72512322e+00,  6.91804826e+00, -4.64325547e+00],
       [ 1.46344371e+02, -1.64285820e+01,  1.93221209e+01, ...,
        -8.02272425e+00,  8.03698208e+00, -1.79963717e+00],
       [ 4.00557034e+01,  8.90352776e+00,  3.74679064e+01, ...,
         7.74476468e+00,  1.16819075e+01,  5.72355186e+00]])

nCol =np.arange(1,len(X_pca[1,:])+1)

col = []
for x in nCol:
    col.append('PC'+str(x))

X_pca_ = pd.DataFrame(data=X_pca,columns=col)

X_pca_.head()

	PC1	PC2	PC3	PC4	PC5	PC6	PC7	PC8	PC9	PC10	...	PC26	PC27	PC28	PC29	PC30	PC31	PC32	PC33	PC34	PC35
0	-164.268682	-104.438239	45.968224	52.386545	-24.668651	-2.737207	-7.430159	5.507118	-30.563076	-24.782297	...	9.883421	1.101969	-4.131897	-13.544494	5.238216	-13.518549	-8.419799	10.993846	1.257210	4.588080
1	7.270394	108.822932	-23.007689	-38.846633	-61.640356	-12.437926	61.213488	-6.071308	36.145302	18.424439	...	2.625201	0.496454	5.804877	-8.400150	1.988566	-6.568501	16.754622	-12.630288	5.771869	0.148876
2	45.553744	56.510936	9.442678	25.585912	11.716666	-43.753559	34.889391	-69.304049	29.532260	-10.061750	...	-7.613603	-9.886747	3.245679	-0.760250	0.957948	1.187809	-4.051917	-2.429993	-7.037015	10.443139
3	-94.974498	103.865795	-1.509163	-49.961214	-38.530131	18.658034	-0.689781	68.761952	-13.524689	8.339692	...	14.636392	-3.947157	-3.055871	-3.878996	-8.006345	-0.237331	0.914040	5.209097	4.532410	-1.561758
4	-28.369577	14.561770	70.761514	-10.352005	-35.642255	-13.865466	-97.641113	11.041376	-19.116691	-3.894344	...	-4.758498	1.717397	2.816929	5.381668	14.067176	-0.710463	10.976279	-11.224621	-26.669710	1.034777

5 rows × 35 columns

One hot Encoding

X[categorical].head()

	x21	x79	x89	x108	x112
0	Jun	Laki-laki	Jambi	SMP	May
1	Jul	Perempuan	Sumatera Barat	SMP	May
2	Jul	Perempuan	Sumatera Selatan	SD	Jun
3	Jun	Laki-laki	Kalimantan Utara	SMP	Sep
4	Jul	Laki-laki	Kalimantan Utara	SMA/SMK	Jun

X_cat = X[categorical]

for i in categorical:
    X_cat = pd.get_dummies(X_cat, columns=[i])

X_cat.head()

	x21_Apr	x21_Aug	x21_Dec	x21_Feb	x21_Jan	x21_Jul	x21_Jun	x21_Mar	x21_May	x21_Nov	...	x112_Dec	x112_Feb	x112_Jan	x112_Jul	x112_Jun	x112_Mar	x112_May	x112_Nov	x112_Oct	x112_Sep
0	0	0	0	0	0	0	1	0	0	0	...	0	0	0	0	0	0	1	0	0	0
1	0	0	0	0	0	1	0	0	0	0	...	0	0	0	0	0	0	1	0	0	0
2	0	0	0	0	0	1	0	0	0	0	...	0	0	0	0	1	0	0	0	0	0
3	0	0	0	0	0	0	1	0	0	0	...	0	0	0	0	0	0	0	0	0	1
4	0	0	0	0	0	1	0	0	0	0	...	0	0	0	0	1	0	0	0	0	0

5 rows × 58 columns

X_nonfloat = X[num_nonfloat]

X_new = pd.concat([X_pca_,X_cat,X_nonfloat], axis=1)

X_new = X_new.drop(labels=['x79_Perempuan'],axis=1)

null_val_sums = X_new.isnull().sum()
pd.set_option("display.max_rows", None)
pd.DataFrame({"Column": null_val_sums.index, "Number of Null Values": null_val_sums.values,
             "Proportion": null_val_sums.values / len(X_new) })

	Column	Number of Null Values	Proportion
0	PC1	0	0.0
1	PC2	0	0.0
2	PC3	0	0.0
3	PC4	0	0.0
4	PC5	0	0.0
5	PC6	0	0.0
6	PC7	0	0.0
7	PC8	0	0.0
8	PC9	0	0.0
9	PC10	0	0.0
10	PC11	0	0.0
11	PC12	0	0.0
12	PC13	0	0.0
13	PC14	0	0.0
14	PC15	0	0.0
15	PC16	0	0.0
16	PC17	0	0.0
17	PC18	0	0.0
18	PC19	0	0.0
19	PC20	0	0.0
20	PC21	0	0.0
21	PC22	0	0.0
22	PC23	0	0.0
23	PC24	0	0.0
24	PC25	0	0.0
25	PC26	0	0.0
26	PC27	0	0.0
27	PC28	0	0.0
28	PC29	0	0.0
29	PC30	0	0.0
30	PC31	0	0.0
31	PC32	0	0.0
32	PC33	0	0.0
33	PC34	0	0.0
34	PC35	0	0.0
35	x21_Apr	0	0.0
36	x21_Aug	0	0.0
37	x21_Dec	0	0.0
38	x21_Feb	0	0.0
39	x21_Jan	0	0.0
40	x21_Jul	0	0.0
41	x21_Jun	0	0.0
42	x21_Mar	0	0.0
43	x21_May	0	0.0
44	x21_Nov	0	0.0
45	x21_Oct	0	0.0
46	x21_Sep	0	0.0
47	x79_Laki-laki	0	0.0
48	x89_Bali	0	0.0
49	x89_Bangka Belitung	0	0.0
50	x89_Banten	0	0.0
51	x89_Bengkulu	0	0.0
52	x89_DI Yogyakarta	0	0.0
53	x89_DKI Jakarta	0	0.0
54	x89_Gorontalo	0	0.0
55	x89_Jambi	0	0.0
56	x89_Jawa Barat	0	0.0
57	x89_Jawa Tengah	0	0.0
58	x89_Jawa Timur	0	0.0
59	x89_Kalimantan Barat	0	0.0
60	x89_Kalimantan Selatan	0	0.0
61	x89_Kalimantan Tengah	0	0.0
62	x89_Kalimantan Timur	0	0.0
63	x89_Kalimantan Utara	0	0.0
64	x89_Maluku	0	0.0
65	x89_Maluku Utara	0	0.0
66	x89_NTB	0	0.0
67	x89_NTT	0	0.0
68	x89_Sulawesi Barat	0	0.0
69	x89_Sulawesi Tengah	0	0.0
70	x89_Sulawesi Tenggara	0	0.0
71	x89_Sulawesi Utara	0	0.0
72	x89_Sumatera Barat	0	0.0
73	x89_Sumatera Selatan	0	0.0
74	x89_Sumatera Utara	0	0.0
75	x108_S1/S2/S3	0	0.0
76	x108_SD	0	0.0
77	x108_SMA/SMK	0	0.0
78	x108_SMP	0	0.0
79	x108_Tidak sekolah	0	0.0
80	x112_Apr	0	0.0
81	x112_Aug	0	0.0
82	x112_Dec	0	0.0
83	x112_Feb	0	0.0
84	x112_Jan	0	0.0
85	x112_Jul	0	0.0
86	x112_Jun	0	0.0
87	x112_Mar	0	0.0
88	x112_May	0	0.0
89	x112_Nov	0	0.0
90	x112_Oct	0	0.0
91	x112_Sep	0	0.0
92	x6	0	0.0
93	x7	0	0.0
94	x22	0	0.0
95	x51	0	0.0
96	x63	0	0.0
97	x80	0	0.0
98	x93	0	0.0
99	x96	0	0.0
100	x104	0	0.0
101	x117	0	0.0
102	x118	0	0.0

X_new.head()

	PC1	PC2	PC3	PC4	PC5	PC6	PC7	PC8	PC9	PC10	...	x7	x22	x51	x63	x80	x93	x96	x104	x117	x118
0	-164.268682	-104.438239	45.968224	52.386545	-24.668651	-2.737207	-7.430159	5.507118	-30.563076	-24.782297	...	2.0	52.0	0.0	13.0	2019.0	2010.0	9.0	1.0	52.0	1.0
1	7.270394	108.822932	-23.007689	-38.846633	-61.640356	-12.437926	61.213488	-6.071308	36.145302	18.424439	...	2.0	78.0	0.0	13.0	2018.0	2009.0	14.0	0.0	53.0	1.0
2	45.553744	56.510936	9.442678	25.585912	11.716666	-43.753559	34.889391	-69.304049	29.532260	-10.061750	...	1.0	39.0	0.0	14.0	2019.0	2009.0	16.0	0.0	53.0	1.0
3	-94.974498	103.865795	-1.509163	-49.961214	-38.530131	18.658034	-0.689781	68.761952	-13.524689	8.339692	...	3.0	48.0	0.0	14.0	2018.0	2014.0	10.0	1.0	50.0	1.0
4	-28.369577	14.561770	70.761514	-10.352005	-35.642255	-13.865466	-97.641113	11.041376	-19.116691	-3.894344	...	1.0	50.0	0.0	14.0	2017.0	2011.0	13.0	1.0	47.0	0.0

5 rows × 103 columns

X[num_nonfloat].head()

	x6	x7	x22	x51	x63	x80	x93	x96	x104	x117	x118
0	4.0	2.0	52.0	0.0	13.0	2019.0	2010.0	9.0	1.0	52.0	1.0
1	4.0	2.0	78.0	0.0	13.0	2018.0	2009.0	14.0	0.0	53.0	1.0
2	5.0	1.0	39.0	0.0	14.0	2019.0	2009.0	16.0	0.0	53.0	1.0
3	3.0	3.0	48.0	0.0	14.0	2018.0	2014.0	10.0	1.0	50.0	1.0
4	3.0	1.0	50.0	0.0	14.0	2017.0	2011.0	13.0	1.0	47.0	0.0

X_train, X_test, y_train, y_test = train_test_split(X_new, y, test_size=0.3, random_state=42)
X_train.shape, X_test.shape

((70000, 103), (30000, 103))

Oversampling

from imblearn.combine import SMOTEENN
from collections import Counter

print('Before', Counter(y_train))

smenn = SMOTEENN()
X_train_smenn, y_train_smenn = smenn.fit_resample(X_train,y_train)

print('after ',Counter(y_train_smenn))

Before Counter({0: 58185, 1: 11815})
after  Counter({1: 57162, 0: 26090})

Feature Selection

VIF

from statsmodels.stats.outliers_influence import variance_inflation_factor

vif_data = pd.DataFrame()
vif_data["feature"] = X_train_smenn.columns
vif_data["VIF"] = [variance_inflation_factor(X_train_smenn.values, i) for i in range(len(X_train_smenn.columns))]
vif_data

	feature	VIF
0	x0	1.002916
1	x1	1.003617
2	x2	1.004175
3	x3	1.003507
4	x4	623.872445
5	x5	1.003461
6	x6	17.581390
7	x7	13.198695
8	x8	1.003526
9	x9	1.004628
10	x10	1.004046
11	x11	1.004346
12	x12	700.793504
13	x13	515.759979
14	x14	1.003314
15	x15	564.456692
16	x16	1.003732
17	x17	675.650176
18	x18	1.002895
19	x19	580.025378
20	x20	1.003613
21	x22	18.382533
22	x23	1.004212
23	x24	1.002748
24	x25	1.003526
25	x26	1.003644
26	x27	1.004452
27	x28	610.962055
28	x29	834.305522
29	x30	1.002902
30	x31	657.560430
31	x32	1.003881
32	x33	1.004469
33	x34	1.003835
34	x35	648.687103
35	x36	592.808805
36	x37	489.770217
37	x38	1.003424
38	x39	588.744693
39	x40	1.003733
40	x41	1109.078481
41	x42	1.003151
42	x43	1.003423
43	x44	589.376698
44	x45	538.121250
45	x46	747.024893
46	x47	1.002878
47	x48	1.003532
48	x49	1.003517
49	x50	592.082204
50	x51	2.021732
51	x52	1.004054
52	x53	596.043888
53	x54	1.003918
54	x55	796.059814
55	x56	1.003046
56	x57	862.523869
57	x58	1.003752
58	x59	549.039130
59	x60	899.441546
60	x61	658.200499
61	x62	717.866352
62	x63	22.429958
63	x64	666.448553
64	x65	568.562556
65	x66	1.003563
66	x67	702.511529
67	x68	563.326508
68	x69	1.004072
69	x70	1.004327
70	x71	699.638090
71	x72	586.514960
72	x73	790.710676
73	x74	1.004032
74	x75	532.513122
75	x76	1.003351
76	x77	1.003278
77	x78	1.003137
78	x80	780018.651255
79	x81	1.003272
80	x82	938.554759
81	x83	1.003589
82	x84	1.003702
83	x85	1.003889
84	x86	614.023418
85	x87	1.003428
86	x88	1.004214
87	x90	1.003753
88	x91	676.289008
89	x92	666.657148
90	x93	763773.405549
91	x94	1.003082
92	x95	1.003128
93	x96	2011.143349
94	x97	1.003078
95	x98	1.004321
96	x99	734.914747
97	x100	1.002964
98	x101	1.003588
99	x102	675.001089
100	x103	503.578807
101	x104	7.666198
102	x105	698.072651
103	x106	626.781222
104	x107	739.047524
105	x109	657.727531
106	x110	1.004155
107	x111	1.003071
108	x113	1.004174
109	x114	1.004249
110	x115	1.004032
111	x116	1.004567
112	x117	19452.759067
113	x118	6.986399
114	x119	491.474161
115	x120	1.003715
116	x121	1.003755
117	x122	803.037996
118	x123	1.004137
119	x124	1.002915
120	x21_Apr	2.013227
121	x21_Aug	2.587533
122	x21_Dec	1.013978
123	x21_Feb	1.064783
124	x21_Jan	1.015754
125	x21_Jul	2.944068
126	x21_Jun	2.829619
127	x21_Mar	1.335678
128	x21_May	2.616829
129	x21_Nov	1.092695
130	x21_Oct	1.319086
131	x21_Sep	1.894016
132	x79_Laki-laki	4.321495
133	x79_Perempuan	4.322053
134	x89_Bali	1.018045
135	x89_Bangka Belitung	1.118506
136	x89_Banten	1.002192
137	x89_Bengkulu	1.217763
138	x89_DI Yogyakarta	1.004924
139	x89_DKI Jakarta	1.001443
140	x89_Gorontalo	1.436639
141	x89_Jambi	1.338964
142	x89_Jawa Barat	1.012742
143	x89_Jawa Tengah	1.024795
144	x89_Jawa Timur	1.052513
145	x89_Kalimantan Barat	2.030053
146	x89_Kalimantan Selatan	1.846400
147	x89_Kalimantan Tengah	2.056642
148	x89_Kalimantan Timur	1.989953
149	x89_Kalimantan Utara	1.638196
150	x89_Maluku	1.002489
151	x89_Maluku Utara	1.001607
152	x89_NTB	1.007321
153	x89_NTT	1.003111
154	x89_Sulawesi Barat	1.269770
155	x89_Sulawesi Tengah	1.154878
156	x89_Sulawesi Tenggara	1.083207
157	x89_Sulawesi Utara	1.035070
158	x89_Sumatera Barat	1.517020
159	x89_Sumatera Selatan	1.745750
160	x89_Sumatera Utara	1.921289
161	x108_S1/S2/S3	1.053834
162	x108_SD	1.990047
163	x108_SMA/SMK	2.520864
164	x108_SMP	4.970641
165	x108_Tidak sekolah	1.022152
166	x112_Apr	1.374906
167	x112_Aug	2.335770
168	x112_Dec	1.002438
169	x112_Feb	1.011000
170	x112_Jan	1.001068
171	x112_Jul	3.020213
172	x112_Jun	2.894283
173	x112_Mar	1.080959
174	x112_May	2.120677
175	x112_Nov	1.018623
176	x112_Oct	1.126160
177	x112_Sep	1.523251

feature = vif_data[vif_data['VIF'] < 5]
feature

	feature	VIF
0	x0	1.002916
1	x1	1.003617
2	x2	1.004175
3	x3	1.003507
5	x5	1.003461
8	x8	1.003526
9	x9	1.004628
10	x10	1.004046
11	x11	1.004346
14	x14	1.003314
16	x16	1.003732
18	x18	1.002895
20	x20	1.003613
22	x23	1.004212
23	x24	1.002748
24	x25	1.003526
25	x26	1.003644
26	x27	1.004452
29	x30	1.002902
31	x32	1.003881
32	x33	1.004469
33	x34	1.003835
37	x38	1.003424
39	x40	1.003733
41	x42	1.003151
42	x43	1.003423
46	x47	1.002878
47	x48	1.003532
48	x49	1.003517
50	x51	2.021732
51	x52	1.004054
53	x54	1.003918
55	x56	1.003046
57	x58	1.003752
65	x66	1.003563
68	x69	1.004072
69	x70	1.004327
73	x74	1.004032
75	x76	1.003351
76	x77	1.003278
77	x78	1.003137
79	x81	1.003272
81	x83	1.003589
82	x84	1.003702
83	x85	1.003889
85	x87	1.003428
86	x88	1.004214
87	x90	1.003753
91	x94	1.003082
92	x95	1.003128
94	x97	1.003078
95	x98	1.004321
97	x100	1.002964
98	x101	1.003588
106	x110	1.004155
107	x111	1.003071
108	x113	1.004174
109	x114	1.004249
110	x115	1.004032
111	x116	1.004567
115	x120	1.003715
116	x121	1.003755
118	x123	1.004137
119	x124	1.002915
120	x21_Apr	2.013227
121	x21_Aug	2.587533
122	x21_Dec	1.013978
123	x21_Feb	1.064783
124	x21_Jan	1.015754
125	x21_Jul	2.944068
126	x21_Jun	2.829619
127	x21_Mar	1.335678
128	x21_May	2.616829
129	x21_Nov	1.092695
130	x21_Oct	1.319086
131	x21_Sep	1.894016
132	x79_Laki-laki	4.321495
133	x79_Perempuan	4.322053
134	x89_Bali	1.018045
135	x89_Bangka Belitung	1.118506
136	x89_Banten	1.002192
137	x89_Bengkulu	1.217763
138	x89_DI Yogyakarta	1.004924
139	x89_DKI Jakarta	1.001443
140	x89_Gorontalo	1.436639
141	x89_Jambi	1.338964
142	x89_Jawa Barat	1.012742
143	x89_Jawa Tengah	1.024795
144	x89_Jawa Timur	1.052513
145	x89_Kalimantan Barat	2.030053
146	x89_Kalimantan Selatan	1.846400
147	x89_Kalimantan Tengah	2.056642
148	x89_Kalimantan Timur	1.989953
149	x89_Kalimantan Utara	1.638196
150	x89_Maluku	1.002489
151	x89_Maluku Utara	1.001607
152	x89_NTB	1.007321
153	x89_NTT	1.003111
154	x89_Sulawesi Barat	1.269770
155	x89_Sulawesi Tengah	1.154878
156	x89_Sulawesi Tenggara	1.083207
157	x89_Sulawesi Utara	1.035070
158	x89_Sumatera Barat	1.517020
159	x89_Sumatera Selatan	1.745750
160	x89_Sumatera Utara	1.921289
161	x108_S1/S2/S3	1.053834
162	x108_SD	1.990047
163	x108_SMA/SMK	2.520864
164	x108_SMP	4.970641
165	x108_Tidak sekolah	1.022152
166	x112_Apr	1.374906
167	x112_Aug	2.335770
168	x112_Dec	1.002438
169	x112_Feb	1.011000
170	x112_Jan	1.001068
171	x112_Jul	3.020213
172	x112_Jun	2.894283
173	x112_Mar	1.080959
174	x112_May	2.120677
175	x112_Nov	1.018623
176	x112_Oct	1.126160
177	x112_Sep	1.523251

selected_features = np.array(feature['feature'])
X_train_v = X_train_smenn[selected_features]
X_test_v = X_test[selected_features]

Pearson Correlation

import seaborn as sns
import matplotlib.pyplot as plt
#Using Pearson Correlation
fig, ax = plt.subplots()
fig.set_size_inches(11,11)
sns.heatmap(X_train_smenn.corr())

<AxesSubplot:>

X_train_smenn.columns.to_frame()

	0
PC1	PC1
PC2	PC2
PC3	PC3
PC4	PC4
PC5	PC5
PC6	PC6
PC7	PC7
PC8	PC8
PC9	PC9
PC10	PC10
PC11	PC11
PC12	PC12
PC13	PC13
PC14	PC14
PC15	PC15
PC16	PC16
PC17	PC17
PC18	PC18
PC19	PC19
PC20	PC20
PC21	PC21
PC22	PC22
PC23	PC23
PC24	PC24
PC25	PC25
PC26	PC26
PC27	PC27
PC28	PC28
PC29	PC29
PC30	PC30
PC31	PC31
PC32	PC32
PC33	PC33
PC34	PC34
PC35	PC35
x21_Apr	x21_Apr
x21_Aug	x21_Aug
x21_Dec	x21_Dec
x21_Feb	x21_Feb
x21_Jan	x21_Jan
x21_Jul	x21_Jul
x21_Jun	x21_Jun
x21_Mar	x21_Mar
x21_May	x21_May
x21_Nov	x21_Nov
x21_Oct	x21_Oct
x21_Sep	x21_Sep
x79_Laki-laki	x79_Laki-laki
x89_Bali	x89_Bali
x89_Bangka Belitung	x89_Bangka Belitung
x89_Banten	x89_Banten
x89_Bengkulu	x89_Bengkulu
x89_DI Yogyakarta	x89_DI Yogyakarta
x89_DKI Jakarta	x89_DKI Jakarta
x89_Gorontalo	x89_Gorontalo
x89_Jambi	x89_Jambi
x89_Jawa Barat	x89_Jawa Barat
x89_Jawa Tengah	x89_Jawa Tengah
x89_Jawa Timur	x89_Jawa Timur
x89_Kalimantan Barat	x89_Kalimantan Barat
x89_Kalimantan Selatan	x89_Kalimantan Selatan
x89_Kalimantan Tengah	x89_Kalimantan Tengah
x89_Kalimantan Timur	x89_Kalimantan Timur
x89_Kalimantan Utara	x89_Kalimantan Utara
x89_Maluku	x89_Maluku
x89_Maluku Utara	x89_Maluku Utara
x89_NTB	x89_NTB
x89_NTT	x89_NTT
x89_Sulawesi Barat	x89_Sulawesi Barat
x89_Sulawesi Tengah	x89_Sulawesi Tengah
x89_Sulawesi Tenggara	x89_Sulawesi Tenggara
x89_Sulawesi Utara	x89_Sulawesi Utara
x89_Sumatera Barat	x89_Sumatera Barat
x89_Sumatera Selatan	x89_Sumatera Selatan
x89_Sumatera Utara	x89_Sumatera Utara
x108_S1/S2/S3	x108_S1/S2/S3
x108_SD	x108_SD
x108_SMA/SMK	x108_SMA/SMK
x108_SMP	x108_SMP
x108_Tidak sekolah	x108_Tidak sekolah
x112_Apr	x112_Apr
x112_Aug	x112_Aug
x112_Dec	x112_Dec
x112_Feb	x112_Feb
x112_Jan	x112_Jan
x112_Jul	x112_Jul
x112_Jun	x112_Jun
x112_Mar	x112_Mar
x112_May	x112_May
x112_Nov	x112_Nov
x112_Oct	x112_Oct
x112_Sep	x112_Sep
x6	x6
x7	x7
x22	x22
x51	x51
x63	x63
x80	x80
x93	x93
x96	x96
x104	x104
x117	x117
x118	x118

def correlation(dataset, threshold):
    col_corr = set()  # Set of all the names of correlated columns
    corr_matrix = dataset.corr()
    for i in range(len(corr_matrix.columns)):
        for j in range(i):
            if abs(corr_matrix.iloc[i, j]) > threshold: # we are interested in absolute coeff value
                colname = corr_matrix.columns[i]  # getting the name of column
                col_corr.add(colname)
    return col_corr

corr_features = correlation(X_train_smenn, 0.7)
len(corr_features)

0

X_train_smenn_m = X_train_smenn.drop(corr_features,axis=1)
X_test_m = X_test.drop(corr_features,axis=1)

Chi-Square

X_cat = X_train[categorical]
X_cat.head()

	x21	x79	x89	x108	x112
76513	Jun	Laki-laki	Kalimantan Utara	SMP	Jul
60406	Jun	Laki-laki	Sumatera Utara	SMA/SMK	Jul
27322	Jun	Laki-laki	Kalimantan Utara	SMP	May
53699	Jul	Laki-laki	Kalimantan Utara	SD	May
65412	Jul	Perempuan	Kalimantan Timur	SMA/SMK	Jul

ordinal_label = {k: i for i, k in enumerate(X_cat['x89'].unique(), 0)}
X_cat['x89'] = X_cat['x89'].map(ordinal_label)

for x in categorical:
    ordinal_label = {k: i for i, k in enumerate(X_cat[x].unique(), 0)}
    X_cat[x] = X_cat[x].map(ordinal_label)
X_cat.head()

	x21	x79	x89	x108	x112
76513	0	0	0	0	0
60406	0	0	1	1	0
27322	0	0	0	0	1
53699	1	0	0	2	1
65412	1	1	2	1	0

from sklearn.feature_selection import chi2
f_p_values=chi2(X_cat,y_train)

p_values=pd.Series(f_p_values[1])
p_values.index=X_cat.columns
p_values

x21     2.217238e-16
x79     8.261815e-01
x89     3.350911e-01
x108    6.244094e-01
x112    1.874501e-01
dtype: float64

from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
chi2_selector = SelectKBest(chi2, k=2)
X_kbest = chi2_selector.fit_transform(X_cat,y_train)
print(X_kbest)

print('Original number of features:', X_cat.shape)
print('Reduced number of features:', X_kbest.shape)

[[0 0]
 [0 0]
 [0 1]
 ...
 [5 3]
 [3 3]
 [5 1]]
Original number of features: (70000, 5)
Reduced number of features: (70000, 2)

X_kbest[:10]

array([[0, 0],
       [0, 0],
       [0, 1],
       [1, 1],
       [1, 0],
       [2, 0],
       [1, 1],
       [1, 2],
       [2, 2],
       [1, 1]], dtype=int64)

nCol=np.arange(1,len(X_kbest[1,:])+1)
col = []
for x in nCol:
    col.append('chi'+str(x))

X_chi = pd.DataFrame(data=X_kbest,columns=col)

print(X_chi.shape,X_train.shape)
# type(X_train)

(70000, 2) (70000, 120)

pandas.core.frame.DataFrame

Model and Evaluation

from sklearn import metrics
from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score

With PCA, Oversampling and One-Hot Encoding

XGBoost

from xgboost import XGBClassifier
xgb = XGBClassifier()
xgb.fit(X_train_smenn, y_train_smenn)

[17:15:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.

XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, enable_categorical=False,
              gamma=0, gpu_id=-1, importance_type=None,
              interaction_constraints='', learning_rate=0.300000012,
              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,
              monotone_constraints='()', n_estimators=100, n_jobs=8,
              num_parallel_tree=1, predictor='auto', random_state=0,
              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
              tree_method='exact', validate_parameters=1, verbosity=None)

y_pred = xgb.predict(X_test)

acc_xgb = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_xgb = recall_score(y_test, y_pred)
tnr_xgb = recall_score(y_test, y_pred, pos_label = 0)
fpr_xgb = 1 - tnr_xgb
prec_xgb = precision_score(y_test, y_pred)
auc_xgb = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's XGBoost's prediction accuracy is: %3.2f" % (acc_xgb))
print("Scikit-Learn's XGBoost's prediction recall is: %3.2f" % (recall_xgb))
print("Scikit-Learn's XGBoost's prediction FPR is: %3.2f" % (fpr_xgb))
print("Scikit-Learn's XGBoost's prediction Precision is: %3.2f" % (prec_xgb))
print("Scikit-Learn's XGBoost's prediction auc_score is: %3.2f" % (auc_xgb))
print("Scikit-Learn's XGBoost's prediction fnr is: %3.2f" % (1-recall_xgb))

cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-xgb.png')

Scikit-Learn's XGBoost's prediction accuracy is: 87.01
Scikit-Learn's XGBoost's prediction recall is: 0.47
Scikit-Learn's XGBoost's prediction FPR is: 0.05
Scikit-Learn's XGBoost's prediction Precision is: 0.68
Scikit-Learn's XGBoost's prediction auc_score is: 0.71
Scikit-Learn's XGBoost's prediction fnr is: 0.53
              precision    recall  f1-score   support

           0       0.90      0.95      0.92     24823
           1       0.68      0.47      0.56      5177

    accuracy                           0.87     30000
   macro avg       0.79      0.71      0.74     30000
weighted avg       0.86      0.87      0.86     30000

y_score = xgb.predict_proba(X_test)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - XGBoost')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')


plt.savefig('ROC_XGB.png',dpi=100)
plt.show()

Random Forest

from sklearn.ensemble import RandomForestClassifier
rfc = RandomForestClassifier(n_estimators=10)
rfc.fit(X_train_smenn, y_train_smenn)
y_pred = rfc.predict(X_test)

acc_rfc = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_rfc = recall_score(y_test, y_pred)
tnr_rfc = recall_score(y_test, y_pred, pos_label = 0)
fpr_rfc = 1 - tnr_rfc
prec_rfc = precision_score(y_test, y_pred)
auc_rfc = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's Random Forest Classifier's prediction accuracy is: %3.2f" % (acc_rfc))
print("Scikit-Learn's Random Forest Classifier's prediction recall is: %3.2f" % (recall_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction FPR is: %3.2f" % (fpr_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction Precision is: %3.2f" % (prec_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction auc_score is: %3.2f" % (auc_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction fnr is: %3.2f" % (1-recall_rfc))

cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-RF.png')

Scikit-Learn's Random Forest Classifier's prediction accuracy is: 80.70
Scikit-Learn's Random Forest Classifier's prediction recall is: 0.34
Scikit-Learn's Random Forests Classifier's prediction FPR is: 0.10
Scikit-Learn's Random Forests Classifier's prediction Precision is: 0.43
Scikit-Learn's Random Forests Classifier's prediction auc_score is: 0.62
Scikit-Learn's Random Forests Classifier's prediction fnr is: 0.66
              precision    recall  f1-score   support

           0       0.87      0.90      0.89     24823
           1       0.43      0.34      0.38      5177

    accuracy                           0.81     30000
   macro avg       0.65      0.62      0.63     30000
weighted avg       0.79      0.81      0.80     30000

y_score = rfc.predict_proba(X_test)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - Random Forest')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')


plt.savefig('ROC_RF.png',dpi=100)
plt.show()

Logistic Regression

from sklearn.linear_model import LogisticRegression
from sklearn import metrics

logreg = LogisticRegression(class_weight='auto',max_iter=10000)
model=logreg.fit(X_train_smenn, y_train_smenn)
y_pred=logreg.predict(X_test)

acc_logreg = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_logreg = recall_score(y_test, y_pred)
tnr_logreg = recall_score(y_test, y_pred, pos_label = 0)
fpr_logreg = 1 - tnr_logreg
prec_logreg = precision_score(y_test, y_pred)
auc_logreg = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's Logistic Regression's prediction accuracy is: %3.2f" % (acc_logreg))
print("Scikit-Learn's Logistic Regression's prediction recall is: %3.2f" % (recall_logreg))
print("Scikit-Learn's Logistic Regression's prediction FPR is: %3.2f" % (fpr_logreg))
print("Scikit-Learn's Logistic Regression's prediction Precision is: %3.2f" % (prec_logreg))
print("Scikit-Learn's Logistic Regression's prediction auc_score is: %3.2f" % (auc_logreg))
print("Scikit-Learn's Logistic Regression's prediction fnr is: %3.2f" % (1-recall_logreg))


cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-logreg.png')

Scikit-Learn's Logistic Regression's prediction accuracy is: 82.22
Scikit-Learn's Logistic Regression's prediction recall is: 0.26
Scikit-Learn's Logistic Regression's prediction FPR is: 0.06
Scikit-Learn's Logistic Regression's prediction Precision is: 0.47
Scikit-Learn's Logistic Regression's prediction auc_score is: 0.60
Scikit-Learn's Logistic Regression's prediction fnr is: 0.74
              precision    recall  f1-score   support

           0       0.86      0.94      0.90     24823
           1       0.47      0.26      0.33      5177

    accuracy                           0.82     30000
   macro avg       0.67      0.60      0.61     30000
weighted avg       0.79      0.82      0.80     30000

y_score = logreg.predict_proba(X_test)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - Logistic Regression')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')

plt.savefig('ROC_Logistic.png',dpi=100)
plt.show()

LightGBM

# !pip install lightgbm

from lightgbm import LGBMClassifier
lgb = LGBMClassifier()
lgb.fit(X_train_smenn, y_train_smenn)
y_pred = lgb.predict(X_test)

acc_lgbm = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_lgbm = recall_score(y_test, y_pred)
tnr_lgbm = recall_score(y_test, y_pred, pos_label = 0)
fpr_lgbm = 1 - tnr_lgbm
prec_lgbm = precision_score(y_test, y_pred)
auc_lgbm = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's LightGBM's prediction accuracy is: %3.2f" % (acc_lgbm))
print("Scikit-Learn's LightGBM's prediction recall is: %3.2f" % (recall_lgbm))
print("Scikit-Learn's LightGBM's prediction FPR is: %3.2f" % (fpr_lgbm))
print("Scikit-Learn's LightGBM's prediction Precision is: %3.2f" % (prec_lgbm))
print("Scikit-Learn's LightGBM's prediction auc_score is: %3.2f" % (auc_lgbm))
print("Scikit-Learn's LightGBM's prediction fnr is: %3.2f" % (1-recall_lgbm))


cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-lgbm.png')

Scikit-Learn's LightGBM's prediction accuracy is: 87.92
Scikit-Learn's LightGBM's prediction recall is: 0.45
Scikit-Learn's LightGBM's prediction FPR is: 0.03
Scikit-Learn's LightGBM's prediction Precision is: 0.75
Scikit-Learn's LightGBM's prediction auc_score is: 0.71
Scikit-Learn's LightGBM's prediction fnr is: 0.55
              precision    recall  f1-score   support

           0       0.89      0.97      0.93     24823
           1       0.75      0.45      0.56      5177

    accuracy                           0.88     30000
   macro avg       0.82      0.71      0.75     30000
weighted avg       0.87      0.88      0.87     30000

y_score = lgb.predict_proba(X_test)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - Logistic Regression')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')

plt.savefig('ROC_Logistic.png',dpi=100)
plt.show()

results = pd.DataFrame({
    'Model': [ 
              'XGBoost', 'Random Forest','Logistic Regression', 'LightGBM'],
    'Recall': [ recall_xgb, recall_rfc,recall_logreg,recall_lgbm],
    'FPR': [ fpr_xgb, fpr_rfc, fpr_logreg,fpr_lgbm],
    'Accuracy': [ acc_xgb, acc_rfc, acc_logreg,acc_lgbm],
    'Precission':[ prec_xgb, prec_rfc, prec_logreg,prec_lgbm],
    'AUC_ROC':[ auc_xgb, auc_rfc, auc_logreg,auc_lgbm],
    'FNR':[1-recall_xgb, 1-recall_rfc, 1-recall_logreg, 1-recall_lgbm]})
result_df = results.sort_values(by='Recall', ascending=False)
result_df = result_df.set_index('Model')
result_df

	Recall	FPR	Accuracy	Precission	AUC_ROC	FNR
Model
XGBoost	0.474020	0.047295	87.010000	0.676406	0.713362	0.525980
LightGBM	0.451420	0.031584	87.920000	0.748798	0.709918	0.548580
Random Forest	0.338034	0.095154	80.703333	0.425584	0.621440	0.661966
Logistic Regression	0.255940	0.059703	82.220000	0.472034	0.598119	0.744060

Tuning Hyperparameter (GridSearchCV)

from sklearn.model_selection import GridSearchCV
params = { 'max_depth': [3,6,10],
           'learning_rate': [0.01, 0.05, 0.1],
           'n_estimators': [100, 500, 1000],
           'colsample_bytree': [0.3, 0.7]}
xgb = XGBClassifier(seed = 20)
clf = GridSearchCV(estimator=xgb, 
                   param_grid=params,
                   scoring='neg_mean_squared_error', 
                   verbose=1)
clf.fit(X_train_smenn, y_train_smenn)
print("Best parameters:", clf.best_params_)
print("Lowest RMSE: ", (-clf.best_score_)**(1/2.0))

Fitting 5 folds for each of 54 candidates, totalling 270 fits
[09:50:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:50:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:50:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:50:13] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:50:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:50:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:50:42] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:51:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:51:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:51:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:51:53] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:52:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:53:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:53:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:54:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:55:06] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:55:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:55:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:55:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:55:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:55:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:56:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:57:14] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:57:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:58:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[09:59:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:00:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:02:04] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:03:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:04:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:06:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:06:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:06:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:06:43] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:06:56] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:07:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:08:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:09:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:10:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:11:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:13:00] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:15:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:17:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:20:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:23:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:26:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:26:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:26:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:26:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:26:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:26:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:27:08] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:27:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:27:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:28:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:28:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:30:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:31:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:31:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:32:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:33:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:33:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:33:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:33:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:33:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:33:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:34:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:35:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:35:57] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:36:37] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:37:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:38:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:40:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:41:25] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:42:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:44:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:44:25] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:44:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:44:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:45:08] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:45:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:46:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:47:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:48:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:49:47] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:50:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:52:46] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:55:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:57:25] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[10:59:46] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:02:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:02:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:02:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:02:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:02:25] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:02:30] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:02:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:03:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:03:42] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:04:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:04:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:05:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:06:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:06:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:07:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:08:13] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:08:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:08:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:08:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:08:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:08:58] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:09:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:10:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:11:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:11:42] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:12:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:13:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:14:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:16:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:17:43] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:19:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:19:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:19:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:19:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:19:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:20:13] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:21:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:22:13] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:23:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:24:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:25:37] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:26:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:29:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:31:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:33:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:35:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:35:53] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:36:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:36:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:36:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:36:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:37:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:37:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:38:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:39:13] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:39:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:41:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:42:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:44:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:46:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:49:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:50:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:51:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:51:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:51:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:52:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:53:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:54:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[11:57:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:00:30] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:02:08] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:04:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:06:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:09:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:11:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:14:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:14:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:15:17] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:15:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:16:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:16:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:18:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:21:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:25:35] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:29:25] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:33:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:39:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:47:42] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[12:55:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:03:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:11:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:11:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:12:04] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:12:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:12:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:12:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:13:58] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:15:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:16:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:17:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:18:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:20:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:23:04] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:25:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:28:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:30:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:31:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:31:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:32:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:32:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:33:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:36:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:38:43] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:41:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:43:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:46:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:49:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:52:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:54:37] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[13:59:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:01:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:02:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:02:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:05:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:06:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:07:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:08:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:12:22] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:15:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:17:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:19:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:21:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:25:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:32:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:40:04] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:47:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:47:37] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:47:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:48:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:48:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:48:32] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:49:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:50:44] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:51:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:52:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:53:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:56:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[14:58:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:00:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:02:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:04:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:05:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:05:42] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:06:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:06:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:07:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:09:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:11:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:13:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:15:27] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:17:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:20:46] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:25:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:29:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:33:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:38:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:39:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:40:14] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:41:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:42:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:43:27] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:44:52] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:46:53] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:49:06] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:51:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:53:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[15:56:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[16:04:27] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[16:12:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[16:20:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[16:30:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
Best parameters: {'colsample_bytree': 0.7, 'learning_rate': 0.01, 'max_depth': 10, 'n_estimators': 500}
Lowest RMSE:  0.35659513427770345

from xgboost import XGBClassifier
xgb = XGBClassifier(**clf.best_params_)
xgb.fit(X_train_smenn, y_train_smenn)

y_pred = xgb.predict(X_test)

[16:35:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.

acc_xgb = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_xgb = recall_score(y_test, y_pred)
tnr_xgb = recall_score(y_test, y_pred, pos_label = 0)
fpr_xgb = 1 - tnr_xgb
prec_xgb = precision_score(y_test, y_pred)
auc_xgb = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's XGBoost's prediction accuracy is: %3.2f" % (acc_xgb))
print("Scikit-Learn's XGBoost's prediction recall is: %3.2f" % (recall_xgb))
print("Scikit-Learn's XGBoost's prediction FPR is: %3.2f" % (fpr_xgb))
print("Scikit-Learn's XGBoost's prediction Precision is: %3.2f" % (prec_xgb))
print("Scikit-Learn's XGBoost's prediction auc_score is: %3.2f" % (auc_xgb))
print("Scikit-Learn's XGBoost's prediction fnr is: %3.2f" % (1-recall_xgb))

cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-xgb.png')

Scikit-Learn's XGBoost's prediction accuracy is: 87.96
Scikit-Learn's XGBoost's prediction recall is: 0.40
Scikit-Learn's XGBoost's prediction FPR is: 0.02
Scikit-Learn's XGBoost's prediction Precision is: 0.81
Scikit-Learn's XGBoost's prediction auc_score is: 0.69
Scikit-Learn's XGBoost's prediction fnr is: 0.60
              precision    recall  f1-score   support

           0       0.89      0.98      0.93     24823
           1       0.81      0.40      0.53      5177

    accuracy                           0.88     30000
   macro avg       0.85      0.69      0.73     30000
weighted avg       0.87      0.88      0.86     30000

y_score = xgb.predict_proba(X_test)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - XGBoost')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')


plt.savefig('ROC_XGB.png',dpi=100)
plt.show()

With VIF and Oversampling

XGBoost

from xgboost import XGBClassifier
xgb = XGBClassifier()
xgb.fit(X_train_v, y_train_smenn)

[14:02:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.

XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, enable_categorical=False,
              gamma=0, gpu_id=-1, importance_type=None,
              interaction_constraints='', learning_rate=0.300000012,
              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,
              monotone_constraints='()', n_estimators=100, n_jobs=8,
              num_parallel_tree=1, predictor='auto', random_state=0,
              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
              tree_method='exact', validate_parameters=1, verbosity=None)

y_pred = xgb.predict(X_test_v)

acc_xgb = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_xgb = recall_score(y_test, y_pred)
tnr_xgb = recall_score(y_test, y_pred, pos_label = 0)
fpr_xgb = 1 - tnr_xgb
prec_xgb = precision_score(y_test, y_pred)
auc_xgb = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's XGBoost's prediction accuracy is: %3.2f" % (acc_xgb))
print("Scikit-Learn's XGBoost's prediction recall is: %3.2f" % (recall_xgb))
print("Scikit-Learn's XGBoost's prediction FPR is: %3.2f" % (fpr_xgb))
print("Scikit-Learn's XGBoost's prediction Precision is: %3.2f" % (prec_xgb))
print("Scikit-Learn's XGBoost's prediction auc_score is: %3.2f" % (auc_xgb))
print("Scikit-Learn's XGBoost's prediction fnr is: %3.2f" % (1-recall_xgb))

cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-xgb.png')

Scikit-Learn's XGBoost's prediction accuracy is: 87.88
Scikit-Learn's XGBoost's prediction recall is: 0.36
Scikit-Learn's XGBoost's prediction FPR is: 0.01
Scikit-Learn's XGBoost's prediction Precision is: 0.85
Scikit-Learn's XGBoost's prediction auc_score is: 0.67
Scikit-Learn's XGBoost's prediction fnr is: 0.64
              precision    recall  f1-score   support

           0       0.88      0.99      0.93     24823
           1       0.85      0.36      0.51      5177

    accuracy                           0.88     30000
   macro avg       0.86      0.67      0.72     30000
weighted avg       0.88      0.88      0.86     30000

y_score = xgb.predict_proba(X_test_v)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - XGBoost')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')


plt.savefig('ROC_XGB.png',dpi=100)
plt.show()

Random Forest

from sklearn.ensemble import RandomForestClassifier
rfc = RandomForestClassifier(n_estimators=10)
rfc.fit(X_train_v, y_train_smenn)
y_pred = rfc.predict(X_test_v)

acc_rfc = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_rfc = recall_score(y_test, y_pred)
tnr_rfc = recall_score(y_test, y_pred, pos_label = 0)
fpr_rfc = 1 - tnr_rfc
prec_rfc = precision_score(y_test, y_pred)
auc_rfc = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's Random Forest Classifier's prediction accuracy is: %3.2f" % (acc_rfc))
print("Scikit-Learn's Random Forest Classifier's prediction recall is: %3.2f" % (recall_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction FPR is: %3.2f" % (fpr_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction Precision is: %3.2f" % (prec_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction auc_score is: %3.2f" % (auc_rfc))
print("Scikit-Learn's Random Forests Classifier's prediction fnr is: %3.2f" % (1-recall_rfc))

cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-RF.png')

Scikit-Learn's Random Forest Classifier's prediction accuracy is: 84.01
Scikit-Learn's Random Forest Classifier's prediction recall is: 0.18
Scikit-Learn's Random Forests Classifier's prediction FPR is: 0.02
Scikit-Learn's Random Forests Classifier's prediction Precision is: 0.63
Scikit-Learn's Random Forests Classifier's prediction auc_score is: 0.58
Scikit-Learn's Random Forests Classifier's prediction fnr is: 0.82
              precision    recall  f1-score   support

           0       0.85      0.98      0.91     24823
           1       0.63      0.18      0.28      5177

    accuracy                           0.84     30000
   macro avg       0.74      0.58      0.59     30000
weighted avg       0.81      0.84      0.80     30000

y_score = rfc.predict_proba(X_test_v)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - Random Forest')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')


plt.savefig('ROC_RF.png',dpi=100)
plt.show()

Logistic Regression

from sklearn.linear_model import LogisticRegression
from sklearn import metrics

logreg = LogisticRegression(class_weight='auto',max_iter=10000)
model=logreg.fit(X_train_v, y_train_smenn)
y_pred=logreg.predict(X_test_v)

acc_logreg = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_logreg = recall_score(y_test, y_pred)
tnr_logreg = recall_score(y_test, y_pred, pos_label = 0)
fpr_logreg = 1 - tnr_logreg
prec_logreg = precision_score(y_test, y_pred)
auc_logreg = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's Logistic Regression's prediction accuracy is: %3.2f" % (acc_logreg))
print("Scikit-Learn's Logistic Regression's prediction recall is: %3.2f" % (recall_logreg))
print("Scikit-Learn's Logistic Regression's prediction FPR is: %3.2f" % (fpr_logreg))
print("Scikit-Learn's Logistic Regression's prediction Precision is: %3.2f" % (prec_logreg))
print("Scikit-Learn's Logistic Regression's prediction auc_score is: %3.2f" % (auc_logreg))
print("Scikit-Learn's Logistic Regression's prediction fnr is: %3.2f" % (1-recall_logreg))


cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-logreg.png')

Scikit-Learn's Logistic Regression's prediction accuracy is: 83.82
Scikit-Learn's Logistic Regression's prediction recall is: 0.10
Scikit-Learn's Logistic Regression's prediction FPR is: 0.01
Scikit-Learn's Logistic Regression's prediction Precision is: 0.72
Scikit-Learn's Logistic Regression's prediction auc_score is: 0.55
Scikit-Learn's Logistic Regression's prediction fnr is: 0.90
              precision    recall  f1-score   support

           0       0.84      0.99      0.91     24823
           1       0.72      0.10      0.18      5177

    accuracy                           0.84     30000
   macro avg       0.78      0.55      0.54     30000
weighted avg       0.82      0.84      0.78     30000

y_score = logreg.predict_proba(X_test_v)[:,1]
false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_score)

plt.title('Receiver Operating Characteristic - Logistic Regression')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')

plt.savefig('ROC_Logistic.png',dpi=100)
plt.show()

results = pd.DataFrame({
    'Model': [ 
              'XGBoost', 'Random Forest','Logistic Regression'],
    'Recall': [ recall_xgb, recall_rfc,recall_logreg],
    'FPR': [ fpr_xgb, fpr_rfc, fpr_logreg],
    'Accuracy': [ acc_xgb, acc_rfc, acc_logreg],
    'Precission':[ prec_xgb, prec_rfc, prec_logreg],
    'AUC_ROC':[ auc_xgb, auc_rfc, auc_logreg],
    'FNR':[1-recall_xgb, 1-recall_rfc, 1-recall_logreg]})
result_df = results.sort_values(by='Recall', ascending=False)
result_df = result_df.set_index('Model')
result_df

	Recall	FPR	Accuracy	Precission	AUC_ROC	FNR
Model
XGBoost	0.362565	0.013496	87.883333	0.848553	0.674535	0.637435
Random Forest	0.176936	0.021633	84.006667	0.630420	0.577652	0.823064
Logistic Regression	0.101990	0.008258	83.820000	0.720327	0.546866	0.898010

ANN

#!pip install tensorflow

Collecting tensorflow
  Downloading tensorflow-2.8.0-cp38-cp38-win_amd64.whl (438.0 MB)
Collecting keras<2.9,>=2.8.0rc0
  Downloading keras-2.8.0-py2.py3-none-any.whl (1.4 MB)
Requirement already satisfied: six>=1.12.0 in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorflow) (1.15.0)
Collecting keras-preprocessing>=1.1.1
  Using cached Keras_Preprocessing-1.1.2-py2.py3-none-any.whl (42 kB)
Collecting tensorflow-io-gcs-filesystem>=0.23.1
  Downloading tensorflow_io_gcs_filesystem-0.23.1-cp38-cp38-win_amd64.whl (1.5 MB)
Collecting flatbuffers>=1.12
  Using cached flatbuffers-2.0-py2.py3-none-any.whl (26 kB)
Collecting gast>=0.2.1
  Downloading gast-0.5.3-py3-none-any.whl (19 kB)
Requirement already satisfied: typing-extensions>=3.6.6 in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorflow) (3.7.4.3)
Collecting termcolor>=1.1.0
  Using cached termcolor-1.1.0.tar.gz (3.9 kB)
Collecting libclang>=9.0.1
  Downloading libclang-13.0.0-py2.py3-none-win_amd64.whl (13.9 MB)
Collecting protobuf>=3.9.2
  Downloading protobuf-3.19.4-cp38-cp38-win_amd64.whl (895 kB)
Collecting opt-einsum>=2.3.2
  Using cached opt_einsum-3.3.0-py3-none-any.whl (65 kB)
Collecting absl-py>=0.4.0
  Using cached absl_py-1.0.0-py3-none-any.whl (126 kB)
Collecting tensorboard<2.9,>=2.8
  Downloading tensorboard-2.8.0-py3-none-any.whl (5.8 MB)
Requirement already satisfied: numpy>=1.20 in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorflow) (1.20.1)
Collecting astunparse>=1.6.0
  Using cached astunparse-1.6.3-py2.py3-none-any.whl (12 kB)
Requirement already satisfied: setuptools in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorflow) (52.0.0.post20210125)
Collecting google-pasta>=0.1.1
  Using cached google_pasta-0.2.0-py3-none-any.whl (57 kB)
Requirement already satisfied: h5py>=2.9.0 in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorflow) (2.10.0)
Collecting grpcio<2.0,>=1.24.3
  Downloading grpcio-1.43.0-cp38-cp38-win_amd64.whl (3.4 MB)
Collecting tf-estimator-nightly==2.8.0.dev2021122109
  Downloading tf_estimator_nightly-2.8.0.dev2021122109-py2.py3-none-any.whl (462 kB)
Requirement already satisfied: wrapt>=1.11.0 in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorflow) (1.12.1)
Requirement already satisfied: wheel<1.0,>=0.23.0 in c:\users\nuzuf\anaconda3\lib\site-packages (from astunparse>=1.6.0->tensorflow) (0.36.2)
Collecting google-auth-oauthlib<0.5,>=0.4.1
  Using cached google_auth_oauthlib-0.4.6-py2.py3-none-any.whl (18 kB)
Requirement already satisfied: werkzeug>=0.11.15 in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (1.0.1)
Collecting markdown>=2.6.8
  Using cached Markdown-3.3.6-py3-none-any.whl (97 kB)
Requirement already satisfied: requests<3,>=2.21.0 in c:\users\nuzuf\anaconda3\lib\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (2.25.1)
Collecting google-auth<3,>=1.6.3
  Downloading google_auth-2.6.0-py2.py3-none-any.whl (156 kB)
Collecting tensorboard-data-server<0.7.0,>=0.6.0
  Using cached tensorboard_data_server-0.6.1-py3-none-any.whl (2.4 kB)
Collecting tensorboard-plugin-wit>=1.6.0
  Using cached tensorboard_plugin_wit-1.8.1-py3-none-any.whl (781 kB)
Collecting rsa<5,>=3.1.4
  Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting pyasn1-modules>=0.2.1
  Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting cachetools<6.0,>=2.0.0
  Downloading cachetools-5.0.0-py3-none-any.whl (9.1 kB)
Collecting requests-oauthlib>=0.7.0
  Downloading requests_oauthlib-1.3.1-py2.py3-none-any.whl (23 kB)
Collecting importlib-metadata>=4.4
  Downloading importlib_metadata-4.10.1-py3-none-any.whl (17 kB)
Requirement already satisfied: zipp>=0.5 in c:\users\nuzuf\anaconda3\lib\site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow) (3.4.1)
Collecting pyasn1<0.5.0,>=0.4.6
  Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: chardet<5,>=3.0.2 in c:\users\nuzuf\anaconda3\lib\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (4.0.0)
Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\users\nuzuf\anaconda3\lib\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (1.26.4)
Requirement already satisfied: certifi>=2017.4.17 in c:\users\nuzuf\anaconda3\lib\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (2020.12.5)
Requirement already satisfied: idna<3,>=2.5 in c:\users\nuzuf\anaconda3\lib\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (2.10)
Collecting oauthlib>=3.0.0
  Downloading oauthlib-3.2.0-py3-none-any.whl (151 kB)
Building wheels for collected packages: termcolor
  Building wheel for termcolor (setup.py): started
  Building wheel for termcolor (setup.py): finished with status 'done'
  Created wheel for termcolor: filename=termcolor-1.1.0-py3-none-any.whl size=4829 sha256=1b842e4f711effc0d03c7dc171710bd17daae8edf64c39086295aecac86741e7
  Stored in directory: c:\users\nuzuf\appdata\local\pip\cache\wheels\a0\16\9c\5473df82468f958445479c59e784896fa24f4a5fc024b0f501
Successfully built termcolor
Installing collected packages: pyasn1, rsa, pyasn1-modules, oauthlib, cachetools, requests-oauthlib, importlib-metadata, google-auth, tensorboard-plugin-wit, tensorboard-data-server, protobuf, markdown, grpcio, google-auth-oauthlib, absl-py, tf-estimator-nightly, termcolor, tensorflow-io-gcs-filesystem, tensorboard, opt-einsum, libclang, keras-preprocessing, keras, google-pasta, gast, flatbuffers, astunparse, tensorflow
  Attempting uninstall: importlib-metadata
    Found existing installation: importlib-metadata 3.10.0
    Uninstalling importlib-metadata-3.10.0:
      Successfully uninstalled importlib-metadata-3.10.0
Successfully installed absl-py-1.0.0 astunparse-1.6.3 cachetools-5.0.0 flatbuffers-2.0 gast-0.5.3 google-auth-2.6.0 google-auth-oauthlib-0.4.6 google-pasta-0.2.0 grpcio-1.43.0 importlib-metadata-4.10.1 keras-2.8.0 keras-preprocessing-1.1.2 libclang-13.0.0 markdown-3.3.6 oauthlib-3.2.0 opt-einsum-3.3.0 protobuf-3.19.4 pyasn1-0.4.8 pyasn1-modules-0.2.8 requests-oauthlib-1.3.1 rsa-4.8 tensorboard-2.8.0 tensorboard-data-server-0.6.1 tensorboard-plugin-wit-1.8.1 tensorflow-2.8.0 tensorflow-io-gcs-filesystem-0.23.1 termcolor-1.1.0 tf-estimator-nightly-2.8.0.dev2021122109

import tensorflow as tf

ann = tf.keras.models.Sequential()

ann.add(tf.keras.layers.Dense(units=177,activation="relu"))

ann.add(tf.keras.layers.Dense(units=177,activation="relu"))

ann.add(tf.keras.layers.Dense(units=1,activation="sigmoid"))

ann.compile(optimizer="adam",loss="binary_crossentropy",metrics=['accuracy'])

ann.fit(X_train,y_train,batch_size=32,epochs = 100)

Epoch 1/100
2188/2188 [==============================] - 5s 2ms/step - loss: 0.4330 - accuracy: 0.8677
Epoch 2/100
2188/2188 [==============================] - 5s 2ms/step - loss: 0.3386 - accuracy: 0.8950
Epoch 3/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.3270 - accuracy: 0.8990
Epoch 4/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.3207 - accuracy: 0.9014
Epoch 5/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.3150 - accuracy: 0.9029
Epoch 6/100
2188/2188 [==============================] - 2s 985us/step - loss: 0.3100 - accuracy: 0.9047
Epoch 7/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.3050 - accuracy: 0.9059
Epoch 8/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.3007 - accuracy: 0.9070
Epoch 9/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2963 - accuracy: 0.9081
Epoch 10/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2913 - accuracy: 0.9091
Epoch 11/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.2855 - accuracy: 0.9106
Epoch 12/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.2795 - accuracy: 0.9121
Epoch 13/100
2188/2188 [==============================] - 2s 940us/step - loss: 0.2736 - accuracy: 0.9132
Epoch 14/100
2188/2188 [==============================] - 2s 843us/step - loss: 0.2673 - accuracy: 0.9157
Epoch 15/100
2188/2188 [==============================] - 4s 2ms/step - loss: 0.2621 - accuracy: 0.9165
Epoch 16/100
2188/2188 [==============================] - 6s 3ms/step - loss: 0.2559 - accuracy: 0.9178
Epoch 17/100
2188/2188 [==============================] - 6s 3ms/step - loss: 0.2504 - accuracy: 0.9197
Epoch 18/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2436 - accuracy: 0.9217
Epoch 19/100
2188/2188 [==============================] - 2s 955us/step - loss: 0.2392 - accuracy: 0.9226
Epoch 20/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.2325 - accuracy: 0.9255
Epoch 21/100
2188/2188 [==============================] - 2s 910us/step - loss: 0.2273 - accuracy: 0.9269
Epoch 22/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2231 - accuracy: 0.9275
Epoch 23/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2170 - accuracy: 0.9298
Epoch 24/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2153 - accuracy: 0.9301
Epoch 25/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.2115 - accuracy: 0.9318
Epoch 26/100
2188/2188 [==============================] - 2s 982us/step - loss: 0.2075 - accuracy: 0.9323
Epoch 27/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2028 - accuracy: 0.9340
Epoch 28/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.2006 - accuracy: 0.9342
Epoch 29/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1963 - accuracy: 0.9357
Epoch 30/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1925 - accuracy: 0.9369
Epoch 31/100
2188/2188 [==============================] - 2s 913us/step - loss: 0.1887 - accuracy: 0.9375
Epoch 32/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1855 - accuracy: 0.9394
Epoch 33/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1844 - accuracy: 0.9395
Epoch 34/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1810 - accuracy: 0.9398
Epoch 35/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1772 - accuracy: 0.9413
Epoch 36/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1770 - accuracy: 0.9413
Epoch 37/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1762 - accuracy: 0.9407
Epoch 38/100
2188/2188 [==============================] - 2s 998us/step - loss: 0.1693 - accuracy: 0.9430
Epoch 39/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1690 - accuracy: 0.9440
Epoch 40/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1665 - accuracy: 0.9444
Epoch 41/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1632 - accuracy: 0.9454
Epoch 42/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1642 - accuracy: 0.9442
Epoch 43/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1610 - accuracy: 0.9453
Epoch 44/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1604 - accuracy: 0.9456
Epoch 45/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1586 - accuracy: 0.9454
Epoch 46/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1553 - accuracy: 0.9462
Epoch 47/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1546 - accuracy: 0.9464
Epoch 48/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1543 - accuracy: 0.9459
Epoch 49/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1509 - accuracy: 0.9472
Epoch 50/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1481 - accuracy: 0.9474
Epoch 51/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1491 - accuracy: 0.9472
Epoch 52/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1469 - accuracy: 0.9478
Epoch 53/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1450 - accuracy: 0.9484
Epoch 54/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1428 - accuracy: 0.9487
Epoch 55/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1417 - accuracy: 0.9489
Epoch 56/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1418 - accuracy: 0.9487
Epoch 57/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1402 - accuracy: 0.9492
Epoch 58/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1387 - accuracy: 0.9490
Epoch 59/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1347 - accuracy: 0.9506
Epoch 60/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1338 - accuracy: 0.9501
Epoch 61/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1340 - accuracy: 0.9503
Epoch 62/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1322 - accuracy: 0.9506
Epoch 63/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1318 - accuracy: 0.9509
Epoch 64/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1289 - accuracy: 0.9511
Epoch 65/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1304 - accuracy: 0.9506
Epoch 66/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1270 - accuracy: 0.9512
Epoch 67/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1248 - accuracy: 0.9522
Epoch 68/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1266 - accuracy: 0.9511
Epoch 69/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1247 - accuracy: 0.9509
Epoch 70/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1190 - accuracy: 0.9527
Epoch 71/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1211 - accuracy: 0.9520
Epoch 72/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1196 - accuracy: 0.9520
Epoch 73/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1194 - accuracy: 0.9524
Epoch 74/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1198 - accuracy: 0.9521
Epoch 75/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1155 - accuracy: 0.9528
Epoch 76/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1165 - accuracy: 0.9529
Epoch 77/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1130 - accuracy: 0.9533
Epoch 78/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1134 - accuracy: 0.9527
Epoch 79/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1127 - accuracy: 0.9526
Epoch 80/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1117 - accuracy: 0.9528
Epoch 81/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1086 - accuracy: 0.9531
Epoch 82/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1088 - accuracy: 0.9532
Epoch 83/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1120 - accuracy: 0.9530
Epoch 84/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1075 - accuracy: 0.9538
Epoch 85/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1039 - accuracy: 0.9562
Epoch 86/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1064 - accuracy: 0.9557
Epoch 87/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.1056 - accuracy: 0.9558
Epoch 88/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1038 - accuracy: 0.9568
Epoch 89/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1055 - accuracy: 0.9558
Epoch 90/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1021 - accuracy: 0.9586
Epoch 91/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.0997 - accuracy: 0.9582
Epoch 92/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1009 - accuracy: 0.9580
Epoch 93/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.0992 - accuracy: 0.9585
Epoch 94/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.0946 - accuracy: 0.9607
Epoch 95/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.1018 - accuracy: 0.9588
Epoch 96/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.0939 - accuracy: 0.9609
Epoch 97/100
2188/2188 [==============================] - 2s 1ms/step - loss: 0.0960 - accuracy: 0.9607
Epoch 98/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.0935 - accuracy: 0.9611
Epoch 99/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.0925 - accuracy: 0.9617
Epoch 100/100
2188/2188 [==============================] - 3s 1ms/step - loss: 0.0933 - accuracy: 0.9619

<keras.callbacks.History at 0x2493af4c160>

y_pred = ann.predict(X_test)

y_pred = np.where(y_pred > 0.5,1,0)

y_pred = y_pred.ravel()

y_pred[:10]

array([0, 1, 0, 1, 0, 0, 0, 1, 0, 0])

y_pred.shape

(30000,)

acc_xgb = (y_pred == y_test).sum().astype(float) / len(y_pred)*100
recall_xgb = recall_score(y_test, y_pred)
tnr_xgb = recall_score(y_test, y_pred, pos_label = 0)
fpr_xgb = 1 - tnr_xgb
prec_xgb = precision_score(y_test, y_pred)
auc_xgb = roc_auc_score(y_test, y_pred)
print("Scikit-Learn's XGBoost's prediction accuracy is: %3.2f" % (acc_xgb))
print("Scikit-Learn's XGBoost's prediction recall is: %3.2f" % (recall_xgb))
print("Scikit-Learn's XGBoost's prediction FPR is: %3.2f" % (fpr_xgb))
print("Scikit-Learn's XGBoost's prediction Precision is: %3.2f" % (prec_xgb))
print("Scikit-Learn's XGBoost's prediction auc_score is: %3.2f" % (auc_xgb))
print("Scikit-Learn's XGBoost's prediction fnr is: %3.2f" % (1-recall_xgb))

cnf_matrix = metrics.confusion_matrix(y_test, y_pred)
print (metrics.classification_report(y_test,y_pred))
sns.heatmap(cnf_matrix,cmap='coolwarm_r',annot=True,linewidth=0.5,fmt='d')
plt.title('Confusion Matrix')
plt.xlabel('Prediksi')
plt.ylabel('Realita')
plt.savefig('cnf-xgb.png')

Scikit-Learn's XGBoost's prediction accuracy is: 84.72
Scikit-Learn's XGBoost's prediction recall is: 0.49
Scikit-Learn's XGBoost's prediction FPR is: 0.08
Scikit-Learn's XGBoost's prediction Precision is: 0.57
Scikit-Learn's XGBoost's prediction auc_score is: 0.71
Scikit-Learn's XGBoost's prediction fnr is: 0.51
              precision    recall  f1-score   support

           0       0.90      0.92      0.91     24823
           1       0.57      0.49      0.53      5177

    accuracy                           0.85     30000
   macro avg       0.73      0.71      0.72     30000
weighted avg       0.84      0.85      0.84     30000

false_positive_rate, true_positive_rate, threshold = roc_curve(y_test, y_pred_prob)

plt.title('Receiver Operating Characteristic - XGBoost')
plt.plot(false_positive_rate, true_positive_rate)
plt.plot([0, 1], ls="--")
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')


plt.savefig('ROC_XGB.png',dpi=100)
plt.show()

with Chi-squared Selection and PCA